#coding:utf-8
import re,hashlib,time,urllib,httplib
from middle.bloomFilterSpider import bloomFilterSpider

class BaiduNewsSpider(bloomFilterSpider):
    """
百度新闻搜索关键字
use case :
a = BaiduNewsSpider("baidu")
a.start("四川大学",1)
    """
    def start(self,key,page):
        self.key = key
        key = {"word":key}
        for i in range(page):
            self.run('http://news.baidu.com/ns?'+urllib.urlencode(key)+'&pn='+str(i*10))
    def extract(self,html):
        # url,pubtime,title,content,searchword,md5
         for sel in html.xpath('//li[@class="result"]'):
            lable = sel.xpath('h3/a')
            info = sel.xpath('string(div[contains(@class,"c-summary")]//p[@class="c-author"]/text())')
            if lable:
                lable = lable[0]
                res = {}
                res["url"] = lable.attrib.get("href","")
                if self.existUrl(res["url"]):
                    continue;
                res["pubtime"] = self.getPubtime(info)
                res["site"] = self.getSite(info)
                res["title"] = self.getInnerText(lable)
                res["content"] = self.opener.openurl(res["url"])
                res["md5"] = hashlib.md5(res["content"]).hexdigest()
                res["searchword"] = self.key
                yield res
    def getSite(self,info):
        info = re.findall(u'([^&]+)  (.+)',info)
        if info and info[0]:
            return info[0][0]
        return ""
    def getPubtime(self,info):
        info = re.findall(u'([^&]+)  (.+)',info)
        if info and info[0]:
            return self.convertTime(info[0][1])
        return "0"
    def convertTime(self,s):
        m = re.match(u"(\d+)分",s)
        if m:
            return int(time.time() - int(m.group(1))*60)
        m = re.match(u"(\d+)小时",s)
        if m:
            return int(time.time() - int(m.group(1))*3600)
        m = re.match(u"(\d+)天",s)
        if m:
            return int(time.time() - int(m.group(1))*86400)
        m = re.match(u"\d+年\d+月\d+日 \d+:\d+",s)
        if m:
            return int(time.mktime(time.strptime(s,u"%Y年%m月%d日 %H:%M")))
        return "0"